"""

This script will take a list of user edits with byte data formatted like, one line per edit, the output of week7/editPrep.py

And output statistics about

for each cohor, and for each lifespan group of that cohort, inlcuding 
amount of work, broken down by kittur's methodology, by date, one for each edit count, time, and bytes changed
proportion of work, broken down by kittur's methodology, by date, one for each edit count, time, and bytes changed      

""" 

import os
import sys
import datetime

import wp_datetime as dt
import wp_namespace as ns

def main():

        inputDir = sys.argv[1]
        outputDir = sys.argv[2]

	START_COHORT = 2003
	END_COHORT = 2012

	END_YEAR = 2012
	CURRENT_CUTOFF = '2011-05-03'

	work = {}
	current_work = {}
	
	for year in range(START_COHORT, END_COHORT + 1):

        	work[year] = {}
		current_work[year] = {}

                for offset in range (0, END_YEAR - year + 1):

                        work[year][offset] = {}
	


        inputFileList = os.listdir(inputDir)

        for inputFileName in inputFileList:

                inputFilePath = inputDir+'/'+inputFileName

                if os.path.isfile(inputFilePath):
                        processFile(inputFilePath, work, current_work, CURRENT_CUTOFF)

	current_work_prop = {}
	for cohort in current_work:
		current_work_prop[cohort] = findProportion(current_work[cohort])

	work_prop = {}
	for cohort in work:
		work_prop[cohort] = {}
		for lifespan in work[cohort]:
			work_prop[cohort][lifespan] = findProportion(work[cohort][lifespan])

	
	for cohort in current_work:
		
			printStats(current_work[cohort], outputDir + '/' + 'cohort'+str(cohort)+'current_count')
			printStats(current_work_prop[cohort], outputDir + '/' + 'cohort'+str(cohort)+'current_prop')
	
	for cohort in work:
		for lifespan in work[cohort]:

			# Make sure its not the last year, because it is always current
			if END_YEAR - lifespan != cohort:
				printStats(work[cohort][lifespan], outputDir + '/' + 'cohort'+str(cohort)+'lifespan'+str(lifespan)+'_count')
                        	printStats(work_prop[cohort][lifespan], outputDir + '/' + 'cohort'+str(cohort)+'lifespan'+str(lifespan)+'_prop')



def printStats(work, basepath):

	header = ['Date', 'Article', 'Article Talk', 'User', 'User Talk', 'Other', 'Maintenance']

	items = ['article', 'article talk', 'user', 'user talk', 'other', 'maintenance']

	editsfile = open(basepath + '_edits.stat', 'w')
	hoursfile = open(basepath + '_hours.stat', 'w')
	bytesfile = open(basepath + '_bytes.stat', 'w')

	editsfile.write('\t'.join(header) + '\n')
	hoursfile.write('\t'.join(header) + '\n')
	bytesfile.write('\t'.join(header) + '\n')

	for date in sorted(work):

		editsout = [date]
		hoursout = [date]
		bytesout = [date]

		for item in items:

			editsout.append(str(work[date][item]['edits']))
			hoursout.append(str(work[date][item]['hours']))
			bytesout.append(str(work[date][item]['bytes']))

		editsfile.write('\t'.join(editsout) + '\n')
                hoursfile.write('\t'.join(hoursout) + '\n')
                bytesfile.write('\t'.join(bytesout) + '\n')

	editsfile.close()
	hoursfile.close()
	bytesfile.close()


def findProportion(data):

	prop = {}
	for date in data:
		initWorkTable(prop, date)

		total_work = {'edits': 0, 'hours': 0.0, 'bytes': 0}
		for item in data[date]:
			for metric in data[date][item]:
				total_work[metric] += data[date][item][metric]

		for item in data[date]:
			for metric in data[date][item]:
				if total_work[metric] == 0:
					prop[date][item][metric] = 0
				else:
					prop[date][item][metric] = float(data[date][item][metric]) / float(total_work[metric])

	return prop
			

def processFile(inputFilePath, work, current_work, CURRENT_CUTOFF):

        print(timestamp() + " Processing " + inputFilePath)

        inputFileHandle = open(inputFilePath, 'r')

	userdata = []

        for line in inputFileHandle:

                fields = parseLine(line)

		if userdata and fields['username'] != userdata[0]['username']:
			addUser(work, current_work, userdata, CURRENT_CUTOFF)
			userdata = []

		userdata.append(fields)

	addUser(work, current_work, userdata, CURRENT_CUTOFF)

	inputFileHandle.close()

        print(timestamp() + " Finished processing " + inputFilePath)


def addUser(work, current_work, userdata, cutoff):

	if dt.before_fast(userdata[-1]['timestamp'], cutoff):
		for line in userdata:
			addWork(work, line)
	else:
		for line in userdata:
			addWorkToCurrent(current_work, line)

def addWorkToCurrent(worktable, fields):

	cohort = fields['cohort']

	#date = simpleDate(fields['timestamp'])
	date = str(int(fields['cohort']) + int(fields['offset']))       # only use year...
	if not date in worktable[cohort]:
		initWorkTable(worktable[cohort], date)

        classification = classifyWork(fields)

        worktable[cohort][date][classification]['edits'] += 1
        worktable[cohort][date][classification]['hours'] += float(fields['duration']) / float(3600)
        worktable[cohort][date][classification]['bytes'] += abs(int(fields['bytes changed']))


def addWork(worktable, fields):

        cohort = fields['cohort']
        lifespan = fields['lifespan']
	
	#date = simpleDate(fields['timestamp'])   
	date = str(int(fields['cohort']) + int(fields['offset']))       # only use year...
	if not date in worktable[cohort][lifespan]:
  		initWorkTable(worktable[cohort][lifespan], date)


	classification = classifyWork(fields)

       	worktable[cohort][lifespan][date][classification]['edits'] += 1
      	worktable[cohort][lifespan][date][classification]['hours'] += float(fields['duration']) / float(3600)
       	worktable[cohort][lifespan][date][classification]['bytes'] += abs(int(fields['bytes changed']))


def classifyWork(fields):

        if fields['revert status'] == 'revert':
                return 'maintenance'

        if fields['namespace'] == 'main':
                return 'article'
        elif fields['namespace'] == 'talk':
                return 'article talk'
        elif fields['namespace'] == 'user':
                return 'user'
        elif fields['namespace'] == 'user talk':
                return 'user talk'
        else:
                return 'other'


def parseLine(line):

        tokens = line.strip().split('\t')

        fields = {}
        fields['username'] = tokens[0]
        fields['timestamp'] = tokens[1]
        fields['offset'] = int(tokens[2])
        fields['lifespan'] = int(tokens[3])
        fields['cohort'] = int(tokens[4])
        fields['namespace'] = tokens[5]
        fields['revert status'] = tokens[6]
        fields['duration'] = float(tokens[7])
        fields['bytes added'] = int(tokens[8])
        fields['bytes removed'] = int(tokens[9])
        fields['bytes changed'] = int(tokens[10])

        return fields

def initWorkTable(work, date):
        """
        Inits a count for a date in the work table. Must be the simple date returned from simpleDate(), not a timestamp.
        """

        work[date] = {}
        work[date]['article'] = {}
        work[date]['article talk'] = {}
        work[date]['user'] = {}
        work[date]['user talk'] = {}
        work[date]['other'] = {}
        work[date]['maintenance'] = {}

        for item in work[date]:
                work[date][item]['edits'] = 0
                work[date][item]['hours'] = 0.0
                work[date][item]['bytes'] = 0


def simpleDate(timestamp):

        return timestamp.split('T')[0]

def timestamp():

        return datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')


if __name__ == '__main__':
        main()

